


#
import requests as req
from bs4 import BeautifulSoup
import json

#分页下载网页内容  新浪中国军情
#https://mil.news.sina.com.cn/roll/index.d.html?cid=57918&page=1
def get_request_page(headers):

    for page_no in range(10):
        url = f"https://mil.news.sina.com.cn/roll/index.d.html?cid=57918&page={page_no+1}"
        text = get_request_text(url , headers)
        if text != None:
            result_data = get_page_data(page_no+1 , text)
            writer_file(f'news/data_news_{page_no+1}.txt' , result_data)
            print(f"page_no={page_no+1}写入成功")
        else:
            print(f"page_no={page_no+1}写入失败")



def get_request_text(url , headers):
    response = req.get(url = url , headers = headers)
    #response = req.get(url = url)
    response.encoding = response.apparent_encoding
    status_code = response.status_code
    print(f"status_code={status_code}")
    if status_code == 200:
        text = response.text
        print(f"text.json={text}")
        return response.text
    else:
        print(f"下载失败")
        return None


def get_page_data(page_no , text):
    result_list = ''
    soup = BeautifulSoup(markup=text, features='html.parser')
    ul_list = soup.findAll(name='ul', attrs={'class': 'linkNews'})
    for ul_item in ul_list:
        li_list = ul_item.findAll(name='li')
        for li_item in li_list:
            a = li_item.find(name='a')
            date = li_item.find(name='span')
            result_list += a.text +' , '+a.get('href')+' , '+ date.text+ '\n'
    #print(result_list)
    return result_list








def json_test():
    # 假设你有一个JSON格式的字符串
    #json_str = '{"name": "John", "age": 30, "city": "New York"}'
    json_str = '{"name": "John", "age": 30, "city": "New York" , "wapurls":[\"https:/finance.sina.cnhy/2024-08-17/detail-incixywh3373290.d.html\"]}'

    # 使用json.loads()函数将字符串转换为Python对象（这里是一个字典）
    data = json.loads(json_str)
    print(data)




def writer_file(path , data):
    with open(file=path , mode='wt' , encoding='utf8') as f:
        f.write(data)



# 中国军情
url = "https://mil.news.sina.com.cn/roll/index.d.html?cid=57918&page=1"
# 设置请求头，使其看起来像是从浏览器发出的
# headers = {
#     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
#     # 可以根据需要添加更多的请求头
# }
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'
}


# json_test()
reponse_text =  get_request_page(headers)


#standings(reponse_text)
# result = get_data_str(reponse_text)
# writer_file('data_news.txt' , result)
# result = get_standings_str(reponse_text)
# writer_file('data.txt' , result)

